{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import jieba\n",
    "import jieba.posseg as psg\n",
    "def load_data(pos=False,corpus_path='corpus.txt'):\n",
    "    doc_list=[]\n",
    "    for line in open(corpus_path,'r',encoding='utf-8'):\n",
    "        content=line.strip()\n",
    "        filter_list=removeWord(cut_list,pos)\n",
    "        doc_list.append(filter_list)\n",
    "    return doc_list\n",
    "def cutWord(sentence,pos=False):\n",
    "    if not pos:\n",
    "        cut_list=jieba.cut(sentence)\n",
    "    else:\n",
    "        cut_list=psg.cut(sentence)\n",
    "    return cut_list\n",
    "def removeWord(seg_list,pos=False):\n",
    "    stop_word_path='stops_list.txt'\n",
    "    stopword_list=[sw.replace('\\n', '')for sw in open(stop_word_path,'r',encoding='utf-8').readlines()]\n",
    "    filter_list=[]\n",
    "    for seg in seg_list:\n",
    "        if not pos:\n",
    "            word=seg\n",
    "            flag='n'\n",
    "    else:\n",
    "        word=seg.word\n",
    "        flag=seg.flag\n",
    "    if not flag.startswith('n'):\n",
    "        continue\n",
    "    if not word in stopword_list and len(word)>1:\n",
    "        filter_list.append(word)\n",
    "    return filter_list\n",
    "def get_tf(word_list):\n",
    "    tf_dic={}\n",
    "    for word in word_list:\n",
    "        tf_dic[word]=tf_dic.get(word,0.0)+1.0\n",
    "    tt_count=len(word_list)\n",
    "    for k, v in tf_dic.item():\n",
    "        tf_dic[k]=float(v) / tt_count\n",
    "    return tf_dic\n",
    "def get_idf(doc_list):"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
